import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 * 美图录网站爬虫
 * jsoup解析
 * okhttp下载
 */
public class MeitulParse {

    public static void main(String[] args) throws IOException {
//        String url = "https://www.meitulu.com/item/4944.html";
        String baseurl = "https://www.meitulu.com";
        String path="D:\\meitulu";
        int timeout=20000;

        //这里表示同步进行的线程数
        ExecutorService exe= Executors.newFixedThreadPool(3);
        //在这里配置起始
        for (int i=4500;i<=4799;i++){
            exe.submit(new MeituRequestTask(baseurl,timeout,i,path));
        }

        exe.shutdown();
        while (true) {
            if (exe.isTerminated()) {
                System.out.println("全部执行完毕！");
                break;
            }
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }

    }
}

class MeituRequestTask implements Runnable{
    String baseurl;
    int timeout;
    int no;
    String path;

    public MeituRequestTask(String baseurl, int timeout, int no, String path) {
        this.baseurl = baseurl;
        this.timeout = timeout;
        this.no = no;
        this.path = path;
    }

    @Override
    public void run() {
        RequestPage requestPage = new RequestPage( baseurl, timeout,no);
        try {
            requestPage.saveToDir(path);
        } catch (IOException e) {
            e.printStackTrace();
        }

//        System.out.println(no+" :"+requestPage.getTitle());
    }
}



class RequestPage {
    String url;
    String baseurl;
    Document document;
    Elements elements;
    int timeout;
    int no;
    String nexturl = "";
    String thisurl = "";
    String title;

    public RequestPage(String baseurl, int timeout,int no) {
        this.url = "https://www.meitulu.com/item/"+no+".html";
        this.baseurl = baseurl;
        this.timeout = timeout;
        this.no=no;
//        System.out.println("正在解析:"+no);
    }

    public String getTitle() {
        try {
            document = Jsoup.parse(new URL(url), timeout);
        } catch (IOException e) {
            System.out.println("链接错误："+url);
            return null;
        }

        title = document.title();
        return title;
    }

    public Elements getAllImgs() {

        try {
            document = Jsoup.parse(new URL(url), timeout);
        } catch (IOException e) {
            System.out.println("链接错误："+url);
            return null;
        }

        title = document.title();
        elements = document.select(".content img");

        while (hasNextPage(document) && notLastPage(document)) {
            nexturl = baseurl + document.select("#pages a:contains(下一页)").first().attr("href");
            try {
                document = Jsoup.parse(new URL(nexturl), timeout);
            } catch (IOException e) {
                System.out.println("链接错误："+nexturl);
            }
            thisurl = nexturl;
            Elements eNext = document.select(".content img");
            elements.addAll(eNext);
        }

        return elements;
    }

    public boolean saveToDir(String path) throws IOException {
        getAllImgs();
        File dir = new File(path + File.separator +"["+no+"]"+ title
                .replace(" ","-")
                .replace("\\","-")
                .replace("/","-"));
        dir.mkdir();
        File img;
        OkHttpClient okHttpClient = new OkHttpClient.Builder()
                .connectTimeout(20, TimeUnit.SECONDS)
                .readTimeout(20,TimeUnit.SECONDS)
                .writeTimeout(20,TimeUnit.SECONDS)
                .build();

        String refer;
        String UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0";
        String host="mtl.ttsqgs.com";
        for (Element e : elements) {
//            System.out.println("正在下载......  "+e.attr("alt")+" "+e.attr("src"));
            refer="https://www.meitulu.com/img.html?img="+e.attr("src");
            Request request = new Request.Builder().url(e.attr("src"))
                    .header("Referer",refer)
                    .header("User-Agent",UserAgent)
                    .header("Host",host)
                    .build();
            Response response = okHttpClient.newCall(request).execute();
            img = new File(dir + File.separator + e.attr("alt")
                    .replace(" ","")
                    .replace("\\","-")
                    .replace("/","-")+ ".jpg");
            FileOutputStream fileOutputStream = new FileOutputStream(img);
            fileOutputStream.write(response.body().bytes());
            fileOutputStream.flush();
            fileOutputStream.close();
        }
        System.out.println(title+"  完成");
        return true;
    }

    private boolean hasNextPage(Document document) {
        return document.select("#pages a:contains(下一页)").size() > 0;
    }

    private boolean notLastPage(Document document) {
        String url = baseurl + document.select("#pages a:contains(下一页)").first().attr("href");
        return !thisurl.equals(url);
    }
}
